package top.doe.spark

import org.apache.spark.rdd.{PairRDDFunctions, RDD}
import org.apache.spark.{SparkConf, SparkContext}

object WordCount {

  def main(args: Array[String]): Unit = {

    System.setProperty("HADOOP_USER_NAME", "root")
    val conf = new SparkConf().setAppName("WordCount").setMaster("local[*]")
    val sc = new SparkContext(conf)
    val lines: RDD[String] = sc.textFile("hdfs://node-1.51doit.cn:8020/data/wc")
    val words: RDD[String] = lines.flatMap(_.split(" "))
    val wordAndOne: RDD[(String, Int)] = words.map((_, 1))
    //val pairRddFunc = new PairRDDFunctions[String, Int](wordAndOne)
    //val reduce: RDD[(String, Int)] = pairRddFunc.reduceByKey(_ + _)
    val reduced: RDD[(String, Int)] = wordAndOne.reduceByKey(_ + _)
    reduced.saveAsTextFile("hdfs://node-1.51doit.cn:8020/out/wc008")
    sc.stop()
  }

}
